From 3ab4c1a9ec7377b88beb237cdeb41e1792aab66f Mon Sep 17 00:00:00 2001
From: Keir Fraser <keir.fraser@citrix.com>
Date: Tue, 22 Sep 2009 09:18:25 +0100
Subject: [PATCH] EPT: More efficient ept_sync_domain().

Rather than always flushing all CPUs, only flush CPUs this domain is
currently active on, and defer flushing other CPUs until this domain
is scheduled onto them (or the domain is destroyed).

Signed-off-by: George Dunlap <george.dunlap@eu.citrix.com>
Signed-off-by: Keir Fraser <keir.fraser@citrix.com>
---
 xen/arch/x86/domain.c              | 13 ++++++++----
 xen/arch/x86/hvm/vmx/vmx.c         | 34 ++++++++++++++++++++++++------
 xen/include/asm-x86/hvm/vmx/vmcs.h |  1 +
 3 files changed, 38 insertions(+), 10 deletions(-)

diff --git a/xen/arch/x86/domain.c b/xen/arch/x86/domain.c
index 9bfac88412..85e0ba0d59 100644
--- a/xen/arch/x86/domain.c
+++ b/xen/arch/x86/domain.c
@@ -1328,6 +1328,15 @@ static void __context_switch(void)
         p->arch.ctxt_switch_from(p);
     }
 
+    /*
+     * Mark this CPU in next domain's dirty cpumasks before calling
+     * ctxt_switch_to(). This avoids a race on things like EPT flushing,
+     * which is synchronised on that function.
+     */
+    if ( p->domain != n->domain )
+        cpu_set(cpu, n->domain->domain_dirty_cpumask);
+    cpu_set(cpu, n->vcpu_dirty_cpumask);
+
     if ( !is_idle_vcpu(n) )
     {
         memcpy(stack_regs,
@@ -1336,10 +1345,6 @@ static void __context_switch(void)
         n->arch.ctxt_switch_to(n);
     }
 
-    if ( p->domain != n->domain )
-        cpu_set(cpu, n->domain->domain_dirty_cpumask);
-    cpu_set(cpu, n->vcpu_dirty_cpumask);
-
     gdt = !is_pv_32on64_vcpu(n) ? per_cpu(gdt_table, cpu) :
                                   per_cpu(compat_gdt_table, cpu);
     if ( need_full_gdt(n) )
diff --git a/xen/arch/x86/hvm/vmx/vmx.c b/xen/arch/x86/hvm/vmx/vmx.c
index d19163faec..3e4620fa0e 100644
--- a/xen/arch/x86/hvm/vmx/vmx.c
+++ b/xen/arch/x86/hvm/vmx/vmx.c
@@ -72,6 +72,7 @@ static void vmx_fpu_dirty_intercept(void);
 static int vmx_msr_read_intercept(struct cpu_user_regs *regs);
 static int vmx_msr_write_intercept(struct cpu_user_regs *regs);
 static void vmx_invlpg_intercept(unsigned long vaddr);
+static void __ept_sync_domain(void *info);
 
 static int vmx_domain_initialise(struct domain *d)
 {
@@ -91,7 +92,8 @@ static int vmx_domain_initialise(struct domain *d)
 
 static void vmx_domain_destroy(struct domain *d)
 {
-    ept_sync_domain(d);
+    if ( d->arch.hvm_domain.hap_enabled )
+        on_each_cpu(__ept_sync_domain, d, 1);
     vmx_free_vlapic_mapping(d);
 }
 
@@ -666,10 +668,21 @@ static void vmx_ctxt_switch_from(struct vcpu *v)
 
 static void vmx_ctxt_switch_to(struct vcpu *v)
 {
+    struct domain *d = v->domain;
+
     /* HOST_CR4 in VMCS is always mmu_cr4_features. Sync CR4 now. */
     if ( unlikely(read_cr4() != mmu_cr4_features) )
         write_cr4(mmu_cr4_features);
 
+    if ( d->arch.hvm_domain.hap_enabled )
+    {
+        unsigned int cpu = smp_processor_id();
+        /* Test-and-test-and-set this CPU in the EPT-is-synced mask. */
+        if ( !cpu_isset(cpu, d->arch.hvm_domain.vmx.ept_synced) &&
+             !cpu_test_and_set(cpu, d->arch.hvm_domain.vmx.ept_synced) )
+            __invept(1, d->arch.hvm_domain.vmx.ept_control.eptp, 0);
+    }
+
     vmx_restore_guest_msrs(v);
     vmx_restore_dr(v);
     vpmu_load(v);
@@ -1216,11 +1229,20 @@ static void __ept_sync_domain(void *info)
 void ept_sync_domain(struct domain *d)
 {
     /* Only if using EPT and this domain has some VCPUs to dirty. */
-    if ( d->arch.hvm_domain.hap_enabled && d->vcpu && d->vcpu[0] )
-    {
-        ASSERT(local_irq_is_enabled());
-        on_each_cpu(__ept_sync_domain, d, 1);
-    }
+    if ( !d->arch.hvm_domain.hap_enabled || !d->vcpu || !d->vcpu[0] )
+        return;
+
+    ASSERT(local_irq_is_enabled());
+
+    /*
+     * Flush active cpus synchronously. Flush others the next time this domain
+     * is scheduled onto them. We accept the race of other CPUs adding to
+     * the ept_synced mask before on_selected_cpus() reads it, resulting in
+     * unnecessary extra flushes, to avoid allocating a cpumask_t on the stack.
+     */
+    d->arch.hvm_domain.vmx.ept_synced = d->domain_dirty_cpumask;
+    on_selected_cpus(&d->arch.hvm_domain.vmx.ept_synced,
+                     __ept_sync_domain, d, 1);
 }
 
 static void __vmx_inject_exception(int trap, int type, int error_code)
diff --git a/xen/include/asm-x86/hvm/vmx/vmcs.h b/xen/include/asm-x86/hvm/vmx/vmcs.h
index 6695e03c04..f7c3c78cf7 100644
--- a/xen/include/asm-x86/hvm/vmx/vmcs.h
+++ b/xen/include/asm-x86/hvm/vmx/vmcs.h
@@ -67,6 +67,7 @@ struct vmx_domain {
         };
         u64 eptp;
     } ept_control;
+    cpumask_t ept_synced;
 };
 
 struct arch_vmx_struct {
-- 
2.30.2